import facade
import requests
import traceback
import os
import sys
import time

from xjlibrary.our_file_dir import BaseDir
bChange = False
StartTime = time.time()
curPath = os.path.dirname(os.path.abspath(__file__))
HEADERS = {'Accept': '*/*',
           'Accept-Encoding': 'gzip, deflate',
           'Connection': 'keep-alive',
           'User-Agent': 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET4.0C; .NET4.0E)', }
sSqlList = list()
proxiesList = list()
nCount = 0
# Proxies = {
#     'http': '127.0.0.1:8087',
#     'https': '127.0.0.1:8087'
# }
mysqlutils = facade.MysqlUtiles(BaseDir.get_new_path(curPath, "db.ini"), "db", facade.get_streamlogger())


# 测试文字输出到文本


def output(value, files='log'):
    f = open(files, 'wb')
    f.write(value)
    f.close()


# 获取代理池

#
# def GetProxies():
#     conn = MajorDbConnect()
#     sProxSql = r'select * from proxy_pool'
#     cur = conn.cursor()
#     lst = list()
#     if conn:
#         try:
#             cur.execute(sProxSql)
#         except:
#             print('* errSql:' + sProxSql)
#             sMsg = '* errSynax:' + traceback.format_exc()
#         lst = list(cur.fetchall())
#
#         cur.close()
#         conn.close()
#
#     return lst
#
#
# # 取一个代理
#
#
# def GetProx():
#     global proxiesList
#
#     bChange = False
#     if len(proxiesList) < 3:
#         lst = GetProxies()
#         proxiesList = list(set(proxiesList) | set(lst))
#
#     prox = proxiesList[0]
#     del proxiesList[0]
#     tmpDict = dict()
#     tmpDict['http'] = prox[0]
#     tmpDict['https'] = prox[0]
#     prox = tmpDict
#
#     return prox

#
# # 数据库链接
# def MajorDbConnect():
#     cf = configparser.ConfigParser()
#     try:
#         cf.read(os.path.join(curPath, 'db.ini'))
#     except:
#         print('读取配置失败:' + traceback.format_exc())
#         return None
#     if cf:
#         db_host = cf.get('db', 'db_host')
#         db_user = cf.get('db', 'db_user')
#         db_pw = cf.get('db', 'db_pw')
#         db_name = cf.get('db', 'db_name')
#         db_charset = cf.get('db', 'db_charset')
#         conn = pymysql.connect(host=db_host, user=db_user,
#                                passwd=db_pw, db=db_name, charset=db_charset)
#         if conn:
#             return conn


# 获取详情页列表
def GetHtml(nNum):
    sSql = "SELECT filename FROM article WHERE stat = 0 AND failcount < 99  LIMIT {nNum}".format(
        nNum=nNum)
    return mysqlutils.SelectFromDB(sSql)
    # conn = MajorDbConnect()
    # sMsg = None
    # taskList = list()
    # if conn:
    #
    #     cur = conn.cursor()
    #     try:
    #         cur.execute(sSql)
    #     except:
    #         print('* errSql:' + sSql)
    #         sMsg = '* errSynax:' + traceback.format_exc()
    #
    #     taskList = cur.fetchall()
    #     cur.close()
    #     conn.close()
    #
    # return taskList


# 下载详情页面


def DownPage(taskList):
    global sSqlList, nCount
    filename = None
    profix = None
    for task in taskList:
        if not task:
            print('* Task Error')
            sys.exit(-1)

        filename = task[0]
        profix = filename[:4]
        # 文件目录名称

        bFlag = False

        pageUrl = 'http://dbpub.cnki.net/grid2008/dbpub/detail.aspx?dbname=%s&filename=%s' % (
            profix, filename)
        (bFlag, html) = RequestDetailPage(pageUrl)

        # 判断是否请求详情页成功 bFlag（1：成功，0：失败）

        sSql = ''
        if not bFlag:
            sSql = "UPDATE article set failcount = failcount + 1 where filename = '{filename}'".format(
                filename=filename)
        else:
            # 创建文件
            dirName = ''
            line = filename + '★' + \
                   html.replace('\0', ' ').replace(
                       '\r', ' ').replace('\n', ' ') + '\n'
            outPathFile = os.path.join(curPath, 'cnki_cg.big_htm')
            with open(outPathFile, mode='a', encoding='utf-8') as f:
                f.write(line)
                nCount += 1
                print('写入文件数量：%d' % nCount)
                print('耗费时间: %d' % int(time.time() - StartTime) + '秒')

            sSql = "UPDATE article set stat = 1 where filename = '{filename}'".format(
                filename=filename)

        sSqlList.append(sSql)
        UpdateDbById()


# 请求详情页
def RequestDetailPage(pageUrl):
    global bChange

    # if bChange:
    #     Proxies = GetProx()

    print('url:' + pageUrl)
    html = ''
    sExMsg = None
    r = None

    try:
        r = requests.get(pageUrl,
                         # proxies=Proxies,
                         headers=HEADERS, timeout=(20, 30))
        # r = requests.get(pageUrl, headers=HEADERS, timeout=20)
        assert r.status_code == requests.codes.ok
        html = r.content.decode('utf-8').strip()
    except:
        sExMsg = '* exMsg:' + traceback.format_exc()
    finally:
        if 'r' in locals() and r:
            r.close()
    if sExMsg:
        print(sExMsg)
        bChange = True
        return False, None

    if html.find(r'【成果完成人】') < 0:
        print('not found 【成果完成人】')
        return False, None

    if html.endswith(r'</html>') < 0:
        print('not found html')
        return False, None

    return True, html


# 更新数据库状态


def UpdateDbById():
    global sSqlList
    mysqlutils.ExeSqlListToDB(sSqlList)
    # conn = MajorDbConnect()
    # cur = conn.cursor()
    # dbMsg = None
    # if conn:
    #     for sql in sSqlList:
    #         try:
    #             cur.execute(sql)
    #         except:
    #             print('*errSql:' + sql)
    #             dbMsg = '*updateError:' + traceback.format_exc()
    #         if dbMsg:
    #             continue
    #     conn.commit()
    #     conn.close()
    sSqlList = list()

#
# # ----------------------------------------------逻辑部分-----------------------
# # ----------------------------------------------下载详情页----------------------
# # 用于串行工作
# def DoWork():
#     proxiesList = GetProxies()
#     # 获取第一个代理
#     Proxies = GetProx()
#     while True:
#         taskList = GetHtml(1000)
#         if not taskList:
#             break
#         DownPage(taskList)
#

def main(logger=None):
    # # 获取代理池
    # proxiesList = GetProxies()
    # # 获取第一个代理
    # Proxies = GetProx()
    while True:
        taskList = GetHtml(1000)
        if not taskList:
            break
        DownPage(taskList)

    # print('DOWN TOTAL:' + str(nCount))


if __name__ == '__main__':
    main()
